# -*- coding: utf-8 -*-

import requests,re,threading,queue,time,os
from urllib import  parse , request

headers1 = {
    # 'content-type':'utf-8',
    # 'Referer': 'http://www.netbian.com/dongman/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
headers2 = {
    # 'content-type':'utf-8',
    # 'Referer': 'http://www.netbian.com/dongman/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'}

class producer(threading.Thread):
    def __init__(self,page_queue,image_queue,path,*args,**kwargs):
        super(producer, self).__init__(*args,**kwargs)
        self.page_queue = page_queue
        self.image_queue = image_queue
        self.path = path
        
    # def get_url_list(self,urls_list,_re):
    #     urls_list_1 = []
    #     for i in urls_list:
    #         resp = requests.get(i,headers=headers1)
    #         resp.encoding='gbk'
    #         text = resp.text
    #         # 把html中的换行符，去掉，也就是替换成空字符串，因为.不能匹配到换行符
    #         text = re.sub('\n', '', text)
    #         for j in re.findall(_re, text):
    #             urls_list_1.append(j[0])
    #     return urls_list_1
            
    def run(self,):
        while not self.page_queue.empty():
            page_url = self.page_queue.get()
            resp = requests.get(page_url,headers1)
            resp.encoding='gbk'
            text = resp.text
            # 把html中的换行符，去掉，也就是替换成空字符串，因为.不能匹配到换行符
            text = re.sub('\n', '', text)
            _re = r'/desk/\d*.htm'
            for i in re.findall(_re,text,re.DOTALL) :
                resp = requests.get(('http://www.netbian.com'+i),headers=headers2)
                resp.encoding='gbk'
                text = resp.text
                text = re.sub('\n', '', text)
                _ree = r'<div.+?endpage.+?<a.{8}desk.+?<img src.{2}(.+?).{2}alt.+?title.+?</a>'
                _rname = r'<div.+?endpage.+?alt.{2}(.+?).{2}title.+?</a>'
                image_url = re.findall(_ree, text)[0]
                image_name = re.findall(_rname, text)[0]
                self.image_queue.put({'image_url':image_url,'image_path':os.path.join(self.path,f'{image_name}.jpg')})
        print('======================================================')
                    
class consumer(threading.Thread):
    def __init__(self,image_queue,*args,**kwargs):
        super(consumer, self).__init__(*args,**kwargs)
        self.image_queue = image_queue
        
    def run(self,):
        while True:
            try:
                image_obj = self.image_queue.get(timeout=10)
                image_url = image_obj.get('image_url')
                image_path = image_obj.get('image_path')
                try:
                    request.urlretrieve(image_url,image_path)
                    print(f'{image_path}已下载完成')
                except:
                    print(f'{image_path}下载失败！')
            except:
                print('----=====--已经全部下载完成--====--啦啦啦---=====--')
                break

def main(s_type,start_page,end_page):
    page_queue = queue.Queue(220)
    #注意这个队列page_queue，一定要大于实际产生的队列数量，否则会一直卡主
    image_queue  = queue.Queue(2000)
    # urls_1 = []
    path = f'H:\desk-paper\{s_type}'
    if not os.path.exists(path):
        os.mkdir(path)
    # urls_1 = []
    for i in range(start_page,end_page):
        if i ==1:
            url = f'http://www.netbian.com/{s_type}/index.htm'
            # urls_1.append(url)
            page_queue.put(url)
        else:
            url = 'http://www.netbian.com/{s_type}/index_{page}.htm'.format(s_type = s_type,page=i)
            # urls_1.append(url)
            page_queue.put(url)
    # for i in urls_1:
    #     page_queue.put(i)
    # print(len(urls_1))
    for x in range(3):
        th = producer(page_queue,image_queue,path,name=f'生产者{x}号')
        th.start()
    for x in range(8):  
        th = consumer(image_queue,name = f'消费者{x}号')
        th.start()

if __name__ == '__main__':
    main('fengjing',40,200)
    